# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request,FormRequest
from csdn_spider.items import CsdnSpiderItem
import time

class CsdnSpider(scrapy.Spider):
    name = 'csdn'
    allowed_domains = ['csdn.net']
    start_urls = ['http://csdn.net/']
    header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"}
    url = "https://edu.csdn.net/courses/k/p"

    def start_requests(self):
        # set start page
        return [Request("https://passport.csdn.net/account/verify", meta={"cookiejar": 1}, callback=self.parse)]

    def parse(self, response):
        #set post infos
        data={
            "gps":"",
            "username":"crazynuonuo",
            "password":"Cj#1234567890",
            "rememberMe":"true",
            "_eventId":"submit",
        }
        print ("登陆中")

        #login
        return [FormRequest.from_response(response,
                                          meta={"cookiejar":response.meta["cookiejar"]},
                                          headers=self.header,
                                          formdata=data,
                                          callback=self.jump2homepage,
                                          )]

    def jump2homepage(self, response):
        #loop to join url address by page number
        for count in range(1,50):
            thisurl = self.url + str(count)
            yield Request(thisurl, callback=self.jump2curpage, meta={"cookiejar": True})

    def jump2curpage(self,response):
        #get current course address from url address
        cur_url = response.xpath("//div[@class='course_dl_list']/a/@href").extract()
        for url_count in range(0,len(cur_url)):
            yield Request(cur_url[url_count], callback=self.getValue, meta={"cookiejar": True})


    def getValue(self,response):
        #set infos to item
        item = CsdnSpiderItem()
        item['courseName']=response.xpath("//h1/a/text()").extract()
        item['stdNum']=response.xpath("//span[@class='num']/text()").extract()
        item['coursePrice'] = response.xpath("//span[@class='mr10']/text()").extract()
        item['courseTimes']= response.xpath("//span[@class='pinfo']/text()").extract()
        item['courseId']= response.xpath("//input[@id='course_id']/@value").extract()
        yield item
